# 功能：数据预处理流程 + 异常值检测 + 数据质量报告 + 去重逻辑

import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
import os
import matplotlib.pyplot as plt
import seaborn as sns


def detect_outliers(df, numeric_features, method='iqr'):
    """
    使用箱线图方法识别数值型字段的异常值
    :param df: 输入DataFrame
    :param numeric_features: 数值列列表
    :param method: 检测方法，默认 'iqr'
    :return: 包含异常统计的字典
    """
    outlier_report = {}

    # 创建输出目录
    os.makedirs("reports", exist_ok=True)

    for col in numeric_features:
        q1 = df[col].quantile(0.25)
        q3 = df[col].quantile(0.75)
        iqr = q3 - q1
        lower_bound = q1 - 1.5 * iqr
        upper_bound = q3 + 1.5 * iqr

        outliers = df[(df[col] < lower_bound) | (df[col] > upper_bound)]
        outlier_count = len(outliers)
        outlier_report[col] = {
            "outlier_count": outlier_count,
            "lower_bound": lower_bound,
            "upper_bound": upper_bound
        }

        print(f" {col} 发现 {outlier_count} 个异常值（阈值：{lower_bound:.2f} ~ {upper_bound:.2f}）")

        # 绘制箱线图并保存
        plt.figure(figsize=(6, 2))
        sns.boxplot(x=df[col])
        plt.title(f"{col} Boxplot")
        plt.tight_layout()
        plt.savefig(f"reports/outliers_{col}.png")
        plt.close()

    return outlier_report


def preprocess_data(df):
    """
    数据预处理流程：
        - 缺失值填充
        - 标准化数值型字段
        - One-Hot编码分类变量
        - 去重处理
        - 异常值检测
    :param df: 原始DataFrame
    :return: 预处理后的DataFrame
    """
    print("【开始】数据预处理...")

    # 显式复制一份，避免后续警告
    df = df.copy()

    # 去重处理
    if df.duplicated().sum() > 0:
        print(f" 检测到 {df.duplicated().sum()} 条重复记录，正在删除...")
        df = df.drop_duplicates()
    else:
        print(" 未发现重复记录")

    # 手动定义分类列
    categorical_features = [
        'Carrier', 'RoundTrip', 'ODPairID', 'LCC_Comp', 'Multi_Airport',
        'Slot', 'Non_Stop', 'OriginCityMarketID', 'DestCityMarketID',
        'OriginAirportID', 'DestAirportID'
    ]

    # 显式转换为字符串类型（避免 FutureWarning）
    df[categorical_features] = df[categorical_features].apply(
        lambda col: col.astype(str, errors='ignore')
    )

    # 自动识别数值列
    numeric_features = df.drop(columns=categorical_features).select_dtypes(include=[np.number]).columns.tolist()

    print(" 分类变量：", categorical_features)
    print(" 数值变量：", numeric_features)

    # 缺失值填充 + 标准化
    numeric_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='median')),
        ('scaler', StandardScaler())])

    # 缺失值填充 + One-Hot编码
    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='constant', fill_value='Unknown')),
        ('onehot', OneHotEncoder(handle_unknown='ignore'))])

    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, numeric_features),
            ('cat', categorical_transformer, categorical_features)])

    processed_array = preprocessor.fit_transform(df)

    # 如果是稀疏矩阵，转换为 numpy array
    try:
        processed_array = processed_array.toarray()
    except AttributeError:
        pass

    # 获取 OneHotEncoder 的特征名
    cat_pipeline = preprocessor.named_transformers_['cat']
    onehot_step = cat_pipeline.named_steps['onehot']
    feature_names_cat = onehot_step.get_feature_names_out().tolist() if hasattr(onehot_step, 'get_feature_names_out') else []

    # 拼接特征名
    feature_names = numeric_features + feature_names_cat

    # 转换为 DataFrame
    df_processed = pd.DataFrame(processed_array, columns=feature_names)
    print(" processed_array shape:", df_processed.shape)

    # 异常值检测
    print("\n 开始异常值检测...")
    detect_outliers(df[numeric_features], numeric_features)

    # 数据质量报告（预处理后）
    print("\n 预处理后数据质量报告：")
    missing_summary = df_processed.isnull().sum()
    missing_rate = (missing_summary / df_processed.shape[0]) * 100
    quality_report = pd.DataFrame({
        'Missing Count': missing_summary,
        'Missing Rate (%)': missing_rate,
        'Unique Count': df_processed.nunique(),
        'Data Type': df_processed.dtypes
    })
    print(quality_report)

    print("【完成】数据预处理完成")
    return df_processed, preprocessor


if __name__ == "__main__":
    raw_df = pd.read_csv("../data/processed/MarketFarePredictionData.csv")
    sample_df = raw_df.head(100_000).copy()

    print(" 正在进行数据预处理...")
    cleaned_df, _ = preprocess_data(sample_df)

    output_path = "../data/preprocessed/cleaned_data.parquet"
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    cleaned_df.to_parquet(output_path)
    print(f" 已保存预处理后的数据到 {output_path}")
